/*! ******************************************************************************
 *
 * Pentaho Data Integration
 *
 * Copyright (C) 2002-2017 by Hitachi Vantara : http://www.pentaho.com
 *
 *******************************************************************************
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ******************************************************************************/

package org.pentaho.di.job.entries.ssh2get;

import java.io.File;
import java.io.FileOutputStream;
import java.util.Iterator;
import java.util.List;
import java.util.Vector;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileType;
import org.pentaho.di.cluster.SlaveServer;
import org.pentaho.di.core.CheckResultInterface;
import org.pentaho.di.core.Const;
import org.pentaho.di.core.util.Utils;
import org.pentaho.di.core.Result;
import org.pentaho.di.core.annotations.JobEntry;
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.encryption.Encr;
import org.pentaho.di.core.exception.KettleDatabaseException;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.exception.KettleXMLException;
import org.pentaho.di.core.variables.VariableSpace;
import org.pentaho.di.core.vfs.KettleVFS;
import org.pentaho.di.core.xml.XMLHandler;
import org.pentaho.di.i18n.BaseMessages;
import org.pentaho.di.job.Job;
import org.pentaho.di.job.JobMeta;
import org.pentaho.di.job.entries.FTPUtils;
import org.pentaho.di.job.entry.JobEntryBase;
import org.pentaho.di.job.entry.JobEntryInterface;
import org.pentaho.di.job.entry.validator.AndValidator;
import org.pentaho.di.job.entry.validator.JobEntryValidatorUtils;
import org.pentaho.di.repository.ObjectId;
import org.pentaho.di.repository.Repository;
import org.pentaho.di.resource.ResourceEntry;
import org.pentaho.di.resource.ResourceEntry.ResourceType;
import org.pentaho.di.resource.ResourceReference;
import org.pentaho.metastore.api.IMetaStore;
import org.w3c.dom.Node;

import com.trilead.ssh2.Connection;
import com.trilead.ssh2.HTTPProxyData;
import com.trilead.ssh2.KnownHosts;
import com.trilead.ssh2.SFTPv3Client;
import com.trilead.ssh2.SFTPv3DirectoryEntry;
import com.trilead.ssh2.SFTPv3FileAttributes;
import com.trilead.ssh2.SFTPv3FileHandle;

/**
 * This defines a SSH2 GET job entry.
 *
 * @author Samatar
 * @since 17-12-2007
 *
 */
@JobEntry(
    id = "SSH2_GET",
    image = "SHG.svg",
    i18nPackageName = "org.pentaho.di.job.entries.ssh2get",
    name = "JobSSH2GET.TypeDesc",
    description = "JobSSH2GET.Tooltip",
    categoryDescription = "i18n:org.pentaho.di.job:JobCategory.Category.Deprecated" )
public class JobEntrySSH2GET extends JobEntryBase implements Cloneable, JobEntryInterface {
  private static Class<?> PKG = JobEntrySSH2GET.class; // for i18n purposes, needed by Translator2!!

  private String serverName;
  private String userName;
  private String password;
  private String serverPort;
  private String ftpDirectory;
  private String localDirectory;
  private String wildcard;
  private boolean onlyGettingNewFiles; /* Don't overwrite files */
  private boolean usehttpproxy;
  private String httpProxyHost;
  private String httpproxyport;
  private String httpproxyusername;
  private String httpProxyPassword;
  private boolean publicpublickey;
  private String keyFilename;
  private String keyFilePass;
  private boolean useBasicAuthentication;
  private String afterFtpPut;
  private String destinationfolder;
  private boolean createdestinationfolder;
  private boolean cachehostkey;
  private int timeout;
  boolean createtargetfolder;
  boolean includeSubFolders;

  static KnownHosts database = new KnownHosts();
  int nbfilestoget = 0;
  int nbgot = 0;
  int nbrerror = 0;

  public JobEntrySSH2GET( String n ) {
    super( n, "" );
    serverName = null;
    publicpublickey = false;
    keyFilename = null;
    keyFilePass = null;
    usehttpproxy = false;
    httpProxyHost = null;
    httpproxyport = null;
    httpproxyusername = null;
    httpProxyPassword = null;
    serverPort = "22";
    useBasicAuthentication = false;
    afterFtpPut = "do_nothing";
    destinationfolder = null;
    includeSubFolders = false;
    createdestinationfolder = false;
    createtargetfolder = false;
    cachehostkey = false;
    timeout = 0;
  }

  public JobEntrySSH2GET() {
    this( "" );
  }

  @Override
  public Object clone() {
    JobEntrySSH2GET je = (JobEntrySSH2GET) super.clone();
    return je;
  }

  @Override
  public String getXML() {
    StringBuffer retval = new StringBuffer( 128 );

    retval.append( super.getXML() );

    retval.append( "      " ).append( XMLHandler.addTagValue( "servername", serverName ) );
    retval.append( "      " ).append( XMLHandler.addTagValue( "username", userName ) );
    retval.append( "      " ).append(
      XMLHandler.addTagValue( "password", Encr.encryptPasswordIfNotUsingVariables( getPassword() ) ) );
    retval.append( "      " ).append( XMLHandler.addTagValue( "serverport", serverPort ) );
    retval.append( "      " ).append( XMLHandler.addTagValue( "ftpdirectory", ftpDirectory ) );
    retval.append( "      " ).append( XMLHandler.addTagValue( "localdirectory", localDirectory ) );
    retval.append( "      " ).append( XMLHandler.addTagValue( "wildcard", wildcard ) );
    retval.append( "      " ).append( XMLHandler.addTagValue( "only_new", onlyGettingNewFiles ) );

    retval.append( "      " ).append( XMLHandler.addTagValue( "usehttpproxy", usehttpproxy ) );
    retval.append( "      " ).append( XMLHandler.addTagValue( "httpproxyhost", httpProxyHost ) );
    retval.append( "      " ).append( XMLHandler.addTagValue( "httpproxyport", httpproxyport ) );
    retval.append( "      " ).append( XMLHandler.addTagValue( "httpproxyusername", httpproxyusername ) );
    retval.append( "      " ).append( XMLHandler.addTagValue( "httpproxypassword", httpProxyPassword ) );

    retval.append( "      " ).append( XMLHandler.addTagValue( "publicpublickey", publicpublickey ) );
    retval.append( "      " ).append( XMLHandler.addTagValue( "keyfilename", keyFilename ) );
    retval.append( "      " ).append( XMLHandler.addTagValue( "keyfilepass", keyFilePass ) );

    retval.append( "      " ).append( XMLHandler.addTagValue( "usebasicauthentication", useBasicAuthentication ) );
    retval.append( "      " ).append( XMLHandler.addTagValue( "afterftpput", afterFtpPut ) );
    retval.append( "      " ).append( XMLHandler.addTagValue( "destinationfolder", destinationfolder ) );
    retval
      .append( "      " ).append( XMLHandler.addTagValue( "createdestinationfolder", createdestinationfolder ) );
    retval.append( "      " ).append( XMLHandler.addTagValue( "cachehostkey", cachehostkey ) );
    retval.append( "      " ).append( XMLHandler.addTagValue( "timeout", timeout ) );
    retval.append( "      " ).append( XMLHandler.addTagValue( "createtargetfolder", createtargetfolder ) );
    retval.append( "      " ).append( XMLHandler.addTagValue( "includeSubFolders", includeSubFolders ) );

    return retval.toString();
  }

  @Override
  public void loadXML( Node entrynode, List<DatabaseMeta> databases, List<SlaveServer> slaveServers,
    Repository rep, IMetaStore metaStore ) throws KettleXMLException {
    try {
      super.loadXML( entrynode, databases, slaveServers );
      serverName = XMLHandler.getTagValue( entrynode, "servername" );
      userName = XMLHandler.getTagValue( entrynode, "username" );
      password = Encr.decryptPasswordOptionallyEncrypted( XMLHandler.getTagValue( entrynode, "password" ) );
      serverPort = XMLHandler.getTagValue( entrynode, "serverport" );
      ftpDirectory = XMLHandler.getTagValue( entrynode, "ftpdirectory" );
      localDirectory = XMLHandler.getTagValue( entrynode, "localdirectory" );
      wildcard = XMLHandler.getTagValue( entrynode, "wildcard" );
      onlyGettingNewFiles = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "only_new" ) );

      usehttpproxy = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "usehttpproxy" ) );
      httpProxyHost = XMLHandler.getTagValue( entrynode, "httpproxyhost" );
      httpproxyport = XMLHandler.getTagValue( entrynode, "httpproxyport" );
      httpproxyusername = XMLHandler.getTagValue( entrynode, "httpproxyusername" );
      httpProxyPassword = XMLHandler.getTagValue( entrynode, "httpproxypassword" );

      publicpublickey = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "publicpublickey" ) );
      keyFilename = XMLHandler.getTagValue( entrynode, "keyfilename" );
      keyFilePass = XMLHandler.getTagValue( entrynode, "keyfilepass" );

      useBasicAuthentication =
        "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "usebasicauthentication" ) );
      afterFtpPut = XMLHandler.getTagValue( entrynode, "afterftpput" );
      destinationfolder = XMLHandler.getTagValue( entrynode, "destinationfolder" );

      createdestinationfolder =
        "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "createdestinationfolder" ) );
      cachehostkey = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "cachehostkey" ) );
      timeout = Const.toInt( XMLHandler.getTagValue( entrynode, "timeout" ), 0 );

      createtargetfolder = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "createtargetfolder" ) );
      includeSubFolders = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "includeSubFolders" ) );

    } catch ( KettleXMLException xe ) {
      throw new KettleXMLException( BaseMessages.getString( PKG, "JobSSH2GET.Log.UnableLoadXML", xe.getMessage() ) );
    }
  }

  @Override
  public void loadRep( Repository rep, IMetaStore metaStore, ObjectId id_jobentry, List<DatabaseMeta> databases,
    List<SlaveServer> slaveServers ) throws KettleException {
    try {
      serverName = rep.getJobEntryAttributeString( id_jobentry, "servername" );
      userName = rep.getJobEntryAttributeString( id_jobentry, "username" );
      password =
        Encr.decryptPasswordOptionallyEncrypted( rep.getJobEntryAttributeString( id_jobentry, "password" ) );
      serverPort = rep.getJobEntryAttributeString( id_jobentry, "serverport" );
      ftpDirectory = rep.getJobEntryAttributeString( id_jobentry, "ftpdirectory" );
      localDirectory = rep.getJobEntryAttributeString( id_jobentry, "localdirectory" );
      wildcard = rep.getJobEntryAttributeString( id_jobentry, "wildcard" );
      onlyGettingNewFiles = rep.getJobEntryAttributeBoolean( id_jobentry, "only_new" );

      usehttpproxy = rep.getJobEntryAttributeBoolean( id_jobentry, "usehttpproxy" );
      httpProxyHost = rep.getJobEntryAttributeString( id_jobentry, "httpproxyhost" );
      httpproxyport = rep.getJobEntryAttributeString( id_jobentry, "httpproxyport" );
      httpproxyusername = rep.getJobEntryAttributeString( id_jobentry, "httpproxyusername" );
      httpProxyPassword = rep.getJobEntryAttributeString( id_jobentry, "httpproxypassword" );

      publicpublickey = rep.getJobEntryAttributeBoolean( id_jobentry, "publicpublickey" );
      keyFilename = rep.getJobEntryAttributeString( id_jobentry, "keyfilename" );
      keyFilePass = rep.getJobEntryAttributeString( id_jobentry, "keyfilepass" );

      useBasicAuthentication = rep.getJobEntryAttributeBoolean( id_jobentry, "usebasicauthentication" );
      afterFtpPut = rep.getJobEntryAttributeString( id_jobentry, "afterftpput" );
      destinationfolder = rep.getJobEntryAttributeString( id_jobentry, "destinationfolder" );

      createdestinationfolder = rep.getJobEntryAttributeBoolean( id_jobentry, "createdestinationfolder" );
      cachehostkey = rep.getJobEntryAttributeBoolean( id_jobentry, "cachehostkey" );
      timeout = (int) rep.getJobEntryAttributeInteger( id_jobentry, "timeout" );

      createtargetfolder = rep.getJobEntryAttributeBoolean( id_jobentry, "createtargetfolder" );
      includeSubFolders = rep.getJobEntryAttributeBoolean( id_jobentry, "includeSubFolders" );

    } catch ( KettleException dbe ) {
      throw new KettleException( BaseMessages.getString(
        PKG, "JobSSH2GET.Log.UnableLoadRep", "" + id_jobentry, dbe.getMessage() ) );
    }
  }

  @Override
  public void saveRep( Repository rep, IMetaStore metaStore, ObjectId id_job ) throws KettleException {
    try {
      rep.saveJobEntryAttribute( id_job, getObjectId(), "servername", serverName );
      rep.saveJobEntryAttribute( id_job, getObjectId(), "username", userName );
      rep.saveJobEntryAttribute( id_job, getObjectId(), "password", Encr
        .encryptPasswordIfNotUsingVariables( password ) );
      rep.saveJobEntryAttribute( id_job, getObjectId(), "serverport", serverPort );
      rep.saveJobEntryAttribute( id_job, getObjectId(), "ftpdirectory", ftpDirectory );
      rep.saveJobEntryAttribute( id_job, getObjectId(), "localdirectory", localDirectory );
      rep.saveJobEntryAttribute( id_job, getObjectId(), "wildcard", wildcard );
      rep.saveJobEntryAttribute( id_job, getObjectId(), "only_new", onlyGettingNewFiles );

      rep.saveJobEntryAttribute( id_job, getObjectId(), "usehttpproxy", usehttpproxy );
      rep.saveJobEntryAttribute( id_job, getObjectId(), "httpproxyhost", httpProxyHost );
      rep.saveJobEntryAttribute( id_job, getObjectId(), "httpproxyport", httpproxyport );
      rep.saveJobEntryAttribute( id_job, getObjectId(), "httpproxyusername", httpproxyusername );
      rep.saveJobEntryAttribute( id_job, getObjectId(), "httpproxypassword", httpProxyPassword );

      rep.saveJobEntryAttribute( id_job, getObjectId(), "publicpublickey", publicpublickey );
      rep.saveJobEntryAttribute( id_job, getObjectId(), "keyfilename", keyFilename );
      rep.saveJobEntryAttribute( id_job, getObjectId(), "keyfilepass", keyFilePass );

      rep.saveJobEntryAttribute( id_job, getObjectId(), "usebasicauthentication", useBasicAuthentication );

      rep.saveJobEntryAttribute( id_job, getObjectId(), "afterftpput", afterFtpPut );
      rep.saveJobEntryAttribute( id_job, getObjectId(), "destinationfolder", destinationfolder );

      rep.saveJobEntryAttribute( id_job, getObjectId(), "createdestinationfolder", createdestinationfolder );
      rep.saveJobEntryAttribute( id_job, getObjectId(), "cachehostkey", cachehostkey );
      rep.saveJobEntryAttribute( id_job, getObjectId(), "timeout", timeout );
      rep.saveJobEntryAttribute( id_job, getObjectId(), "createtargetfolder", createtargetfolder );
      rep.saveJobEntryAttribute( id_job, getObjectId(), "includeSubFolders", includeSubFolders );
    } catch ( KettleDatabaseException dbe ) {
      throw new KettleException( BaseMessages.getString( PKG, "JobSSH2GET.Log.UnableSaveRep", "" + id_job, dbe
        .getMessage() ) );
    }
  }

  /**
   * @return Returns the directory.
   */
  public String getFtpDirectory() {
    return ftpDirectory;
  }

  /**
   * @param directory
   *          The directory to set.
   */
  public void setFtpDirectory( String directory ) {
    this.ftpDirectory = directory;
  }

  /**
   * @return Returns the password.
   */
  public String getPassword() {
    return password;
  }

  /**
   * @param password
   *          The password to set.
   */
  public void setPassword( String password ) {
    this.password = password;
  }

  /**
   * @return Returns the afterftpput.
   */
  public String getAfterFTPPut() {
    return afterFtpPut;
  }

  /**
   * @param afterFtpPut
   *          The action after (FTP/SSH) transfer to execute
   */
  public void setAfterFTPPut( String afterFtpPut ) {
    this.afterFtpPut = afterFtpPut;
  }

  /**
   * @param proxyPassword
   *          The httpproxypassword to set.
   */
  public void setHTTPProxyPassword( String proxyPassword ) {
    this.httpProxyPassword = proxyPassword;
  }

  /**
   * @return Returns the password.
   */
  public String getHTTPProxyPassword() {
    return httpProxyPassword;
  }

  /**
   * @param keyFilePass
   *          The key file pass to set.
   */
  public void setKeyFilePass( String keyFilePass ) {
    this.keyFilePass = keyFilePass;
  }

  /**
   * @return Returns the key file pass.
   */
  public String getKeyFilePass() {
    return keyFilePass;
  }

  /**
   * @return Returns the serverName.
   */
  public String getServerName() {
    return serverName;
  }

  /**
   * @param serverName
   *          The serverName to set.
   */
  public void setServerName( String serverName ) {
    this.serverName = serverName;
  }

  /**
   * @param proxyhost
   *          The httpproxyhost to set.
   */
  public void setHTTPProxyHost( String proxyhost ) {
    this.httpProxyHost = proxyhost;
  }

  /**
   * @return Returns the HTTP proxy host.
   */
  public String getHTTPProxyHost() {
    return httpProxyHost;
  }

  /**
   * @param keyfilename
   *          The key filename to set.
   */
  public void setKeyFilename( String keyfilename ) {
    this.keyFilename = keyfilename;
  }

  /**
   * @return Returns the key filename.
   */
  public String getKeyFilename() {
    return keyFilename;
  }

  /**
   * @return Returns the userName.
   */
  public String getUserName() {
    return userName;
  }

  /**
   * @param userName
   *          The userName to set.
   */
  public void setUserName( String userName ) {
    this.userName = userName;
  }

  /**
   * @param proxyusername
   *          The httpproxyusername to set.
   */
  public void setHTTPProxyUsername( String proxyusername ) {
    this.httpproxyusername = proxyusername;
  }

  /**
   * @return Returns the userName.
   */
  public String getHTTPProxyUsername() {
    return httpproxyusername;
  }

  /**
   * @return Returns the wildcard.
   */
  public String getWildcard() {
    return wildcard;
  }

  /**
   * @param wildcard
   *          The wildcard to set.
   */
  public void setWildcard( String wildcard ) {
    this.wildcard = wildcard;
  }

  /**
   * @return Returns the localDirectory.
   */
  public String getlocalDirectory() {
    return localDirectory;
  }

  /**
   * @param localDirectory
   *          The localDirectory to set.
   */
  public void setlocalDirectory( String localDirectory ) {
    this.localDirectory = localDirectory;
  }

  /**
   * @return Returns the onlyGettingNewFiles.
   */
  public boolean isOnlyGettingNewFiles() {
    return onlyGettingNewFiles;
  }

  /**
   * @param onlyGettingNewFiles
   *          The onlyGettingNewFiles to set.
   */
  public void setOnlyGettingNewFiles( boolean onlyGettingNewFiles ) {
    this.onlyGettingNewFiles = onlyGettingNewFiles;
  }

  /**
   * @param cachehostkeyin
   *          The cachehostkey to set.
   */
  public void setCacheHostKey( boolean cachehostkeyin ) {
    this.cachehostkey = cachehostkeyin;
  }

  /**
   * @return Returns the cachehostkey.
   */
  public boolean isCacheHostKey() {
    return cachehostkey;
  }

  /**
   * @param httpproxy
   *          The usehttpproxy to set.
   */
  public void setUseHTTPProxy( boolean httpproxy ) {
    this.usehttpproxy = httpproxy;
  }

  /**
   * @return Returns the usehttpproxy.
   */
  public boolean isUseHTTPProxy() {
    return usehttpproxy;
  }

  /**
   * @return Returns the use basic authentication flag.
   */
  public boolean isUseBasicAuthentication() {
    return useBasicAuthentication;
  }

  /**
   * @param useBasicAuthentication
   *          The use basic authentication flag to set.
   */
  public void setUseBasicAuthentication( boolean useBasicAuthentication ) {
    this.useBasicAuthentication = useBasicAuthentication;
  }

  /**
   * @param includeSubFolders
   *          The include sub folders flag to set.
   */
  public void setIncludeSubFolders( boolean includeSubFolders ) {
    this.includeSubFolders = includeSubFolders;
  }

  /**
   * @return Returns the include sub folders flag.
   */
  public boolean isIncludeSubFolders() {
    return includeSubFolders;
  }

  /**
   * @param createdestinationfolderin
   *          The createdestinationfolder to set.
   */
  public void setCreateDestinationFolder( boolean createdestinationfolderin ) {
    this.createdestinationfolder = createdestinationfolderin;
  }

  /**
   * @return Returns the createdestinationfolder.
   */
  public boolean isCreateDestinationFolder() {
    return createdestinationfolder;
  }

  /**
   * @return Returns the CreateTargetFolder.
   */
  public boolean isCreateTargetFolder() {
    return createtargetfolder;
  }

  /**
   * @param createtargetfolderin
   *          The createtargetfolder to set.
   */
  public void setCreateTargetFolder( boolean createtargetfolderin ) {
    this.createtargetfolder = createtargetfolderin;
  }

  /**
   * @param publickey
   *          The publicpublickey to set.
   */
  public void setUsePublicKey( boolean publickey ) {
    this.publicpublickey = publickey;
  }

  /**
   * @return Returns the usehttpproxy.
   */
  public boolean isUsePublicKey() {
    return publicpublickey;
  }

  public String getServerPort() {
    return serverPort;
  }

  public void setServerPort( String serverPort ) {
    this.serverPort = serverPort;
  }

  public void setHTTPProxyPort( String proxyport ) {
    this.httpproxyport = proxyport;
  }

  public String getHTTPProxyPort() {
    return httpproxyport;
  }

  public void setDestinationFolder( String destinationfolderin ) {
    this.destinationfolder = destinationfolderin;
  }

  public String getDestinationFolder() {
    return destinationfolder;
  }

  /**
   * @param timeout
   *          The timeout to set.
   */
  public void setTimeout( int timeout ) {
    this.timeout = timeout;
  }

  /**
   * @return Returns the timeout.
   */
  public int getTimeout() {
    return timeout;
  }

  @Override
  public Result execute( Result previousResult, int nr ) {
    Result result = previousResult;
    result.setResult( false );

    if ( log.isRowLevel() ) {
      logRowlevel( BaseMessages.getString( PKG, "JobSSH2GET.Log.GettingFieldsValue" ) );
    }

    // Get real variable value
    String realServerName = environmentSubstitute( serverName );
    int realServerPort = Const.toInt( environmentSubstitute( serverPort ), 22 );
    String realUserName = environmentSubstitute( userName );
    String realServerPassword = Encr.decryptPasswordOptionallyEncrypted( environmentSubstitute( password ) );
    // Proxy Host
    String realProxyHost = environmentSubstitute( httpProxyHost );
    int realProxyPort = Const.toInt( environmentSubstitute( httpproxyport ), 22 );
    String realproxyUserName = environmentSubstitute( httpproxyusername );
    String realProxyPassword =
      Encr.decryptPasswordOptionallyEncrypted( environmentSubstitute( httpProxyPassword ) );
    // Key file
    String realKeyFilename = environmentSubstitute( keyFilename );
    String relKeyFilepass = environmentSubstitute( keyFilePass );
    // target files
    String realLocalDirectory = environmentSubstitute( localDirectory );
    String realwildcard = environmentSubstitute( wildcard );
    // Remote source
    String realftpDirectory = environmentSubstitute( ftpDirectory );
    // Destination folder (Move to)
    String realDestinationFolder = environmentSubstitute( destinationfolder );

    try {
      // Remote source
      realftpDirectory = FTPUtils.normalizePath( realftpDirectory );
      // Destination folder (Move to)
      realDestinationFolder = FTPUtils.normalizePath( realDestinationFolder );
    } catch ( Exception e ) {
      logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.CanNotNormalizePath", e.getMessage() ) );
      result.setNrErrors( 1 );
      return result;
    }

    // Check for mandatory fields
    if ( log.isRowLevel() ) {
      logRowlevel( BaseMessages.getString( PKG, "JobSSH2GET.Log.CheckingMandatoryFields" ) );
    }

    boolean mandatoryok = true;
    if ( Utils.isEmpty( realServerName ) ) {
      mandatoryok = false;
      logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.ServernameMissing" ) );
    }
    if ( usehttpproxy ) {
      if ( Utils.isEmpty( realProxyHost ) ) {
        mandatoryok = false;
        logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.HttpProxyhostMissing" ) );
      }
    }
    if ( publicpublickey ) {
      if ( Utils.isEmpty( realKeyFilename ) ) {
        mandatoryok = false;
        logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.KeyFileMissing" ) );
      } else {
        // Let's check if key file exists...
        if ( !new File( realKeyFilename ).exists() ) {
          mandatoryok = false;
          logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.KeyFileNotExist" ) );
        }
      }
    }

    if ( Utils.isEmpty( realLocalDirectory ) ) {
      mandatoryok = false;
      logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.LocalFolderMissing" ) );
    } else {
      // Check if target folder exists...
      if ( !new File( realLocalDirectory ).exists() ) {

        if ( createtargetfolder ) {
          // Create Target folder
          if ( !CreateFolder( realLocalDirectory ) ) {
            mandatoryok = false;
          }

        } else {
          mandatoryok = false;
          logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.LocalFolderNotExists", realLocalDirectory ) );
        }
      } else {
        if ( !new File( realLocalDirectory ).isDirectory() ) {
          mandatoryok = false;
          logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.LocalFolderNotFolder", realLocalDirectory ) );
        }
      }
    }
    if ( afterFtpPut.equals( "move_file" ) ) {
      if ( Utils.isEmpty( realDestinationFolder ) ) {
        mandatoryok = false;
        logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.DestinatFolderMissing" ) );
      }
    }

    if ( mandatoryok ) {

      Connection conn = null;
      SFTPv3Client client = null;
      boolean good = true;

      try {
        // Create a connection instance
        conn =
          getConnection(
            realServerName, realServerPort, realProxyHost, realProxyPort, realproxyUserName, realProxyPassword );
        if ( log.isDetailed() ) {
          logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.ConnectionInstanceCreated" ) );
        }
        if ( timeout > 0 ) {
          // Use timeout
          // Cache Host Key
          if ( cachehostkey ) {
            conn.connect( new SimpleVerifier( database ), 0, timeout * 1000 );
          } else {
            conn.connect( null, 0, timeout * 1000 );
          }

        } else {
          // Cache Host Key
          if ( cachehostkey ) {
            conn.connect( new SimpleVerifier( database ) );
          } else {
            conn.connect();
          }
        }

        // Authenticate

        boolean isAuthenticated = false;
        if ( publicpublickey ) {
          isAuthenticated =
            conn.authenticateWithPublicKey( realUserName, new File( realKeyFilename ), relKeyFilepass );
        } else {
          isAuthenticated = conn.authenticateWithPassword( realUserName, realServerPassword );
        }

        // LET'S CHECK AUTHENTICATION ...
        if ( isAuthenticated == false ) {
          logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.AuthenticationFailed" ) );
        } else {
          if ( log.isBasic() ) {
            logBasic( BaseMessages.getString( PKG, "JobSSH2GET.Log.Connected", serverName, userName ) );
          }

          client = new SFTPv3Client( conn );

          if ( log.isDetailed() ) {
            logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.ProtocolVersion", ""
              + client.getProtocolVersion() ) );
          }

          // Check if ftp (source) directory exists
          if ( !Utils.isEmpty( realftpDirectory ) ) {
            if ( !sshDirectoryExists( client, realftpDirectory ) ) {
              good = false;
              logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.RemoteDirectoryNotExist", realftpDirectory ) );
            } else if ( log.isDetailed() ) {
              logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.RemoteDirectoryExist", realftpDirectory ) );
            }
          }

          if ( !Utils.isEmpty( realDestinationFolder ) ) {
            // Check now destination folder
            if ( !sshDirectoryExists( client, realDestinationFolder ) ) {
              if ( createdestinationfolder ) {
                if ( !CreateRemoteFolder( client, realDestinationFolder ) ) {
                  good = false;
                }
              } else {
                good = false;
                logError( BaseMessages.getString(
                  PKG, "JobSSH2GET.Log.DestinatFolderNotExist", realDestinationFolder ) );
              }
            }
          }

          if ( good ) {
            Pattern pattern = null;
            if ( !Utils.isEmpty( realwildcard ) ) {
              pattern = Pattern.compile( realwildcard );
            }

            if ( includeSubFolders ) {
              if ( log.isDetailed() ) {
                logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.RecursiveModeOn" ) );
              }
              copyRecursive( realftpDirectory, realLocalDirectory, client, pattern, parentJob );
            } else {
              if ( log.isDetailed() ) {
                logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.RecursiveModeOff" ) );
              }
              GetFiles( realftpDirectory, realLocalDirectory, client, pattern, parentJob );
            }

            /******************************** RESULT ********************/
            if ( log.isDetailed() ) {
              logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.Result.JobEntryEnd1" ) );
              logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.Result.TotalFiles", "" + nbfilestoget ) );
              logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.Result.TotalFilesPut", "" + nbgot ) );
              logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.Result.TotalFilesError", "" + nbrerror ) );
              logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.Result.JobEntryEnd2" ) );
            }
            if ( nbrerror == 0 ) {
              result.setResult( true );
              /******************************** RESULT ********************/
            }
          }

        }

      } catch ( Exception e ) {
        result.setNrErrors( nbrerror );
        logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.Error.ErrorFTP", e.getMessage() ) );
      } finally {
        if ( conn != null ) {
          conn.close();
        }
        if ( client != null ) {
          client.close();
        }
      }
    }

    return result;
  }

  private Connection getConnection( String servername, int serverport, String proxyhost, int proxyport,
    String proxyusername, String proxypassword ) {
    /* Create a connection instance */

    Connection conn = new Connection( servername, serverport );

    /* We want to connect through a HTTP proxy */
    if ( usehttpproxy ) {
      conn.setProxyData( new HTTPProxyData( proxyhost, proxyport ) );

      /* Now connect */
      // if the proxy requires basic authentication:
      if ( useBasicAuthentication ) {
        conn.setProxyData( new HTTPProxyData( proxyhost, proxyport, proxyusername, proxypassword ) );
      }
    }

    return conn;
  }

  /**
   * Check existence of a file
   *
   * @param sftpClient
   * @param filename
   * @return true, if file exists
   * @throws Exception
   */
  public boolean sshFileExists( SFTPv3Client sftpClient, String filename ) {

    try {
      SFTPv3FileAttributes attributes = sftpClient.stat( filename );

      if ( attributes != null ) {
        return ( attributes.isRegularFile() );
      } else {
        return false;
      }

    } catch ( Exception e ) {
      return false;
    }
  }

  /**
   * Check existence of a local file
   *
   * @param filename
   * @return true, if file exists
   */
  public boolean FileExists( String filename ) {

    FileObject file = null;
    try {
      file = KettleVFS.getFileObject( filename, this );
      if ( !file.exists() ) {
        return false;
      } else {
        if ( file.getType() == FileType.FILE ) {
          return true;
        } else {
          return false;
        }
      }
    } catch ( Exception e ) {
      return false;
    }
  }

  /**
   * Checks if file is a directory
   *
   * @param sftpClient
   * @param filename
   * @return true, if filename is a directory
   */
  public boolean isDirectory( SFTPv3Client sftpClient, String filename ) {
    try {
      return sftpClient.stat( filename ).isDirectory();
    } catch ( Exception e ) {
      // Ignore errors
    }
    return false;
  }

  /**
   * Checks if a directory exists
   *
   * @param sftpClient
   * @param directory
   * @return true, if directory exists
   */
  public boolean sshDirectoryExists( SFTPv3Client sftpClient, String directory ) {
    try {
      SFTPv3FileAttributes attributes = sftpClient.stat( directory );

      if ( attributes != null ) {
        return ( attributes.isDirectory() );
      } else {
        return false;
      }

    } catch ( Exception e ) {
      return false;
    }
  }

  /**
   * Returns the file size of a file
   *
   * @param sftpClient
   * @param filename
   * @return the size of the file
   * @throws Exception
   */
  public long getFileSize( SFTPv3Client sftpClient, String filename ) throws Exception {
    return sftpClient.stat( filename ).size.longValue();
  }

  /**********************************************************
   *
   * @param selectedfile
   * @param wildcard
   * @param pattern
   * @return True if the selectedfile matches the wildcard
   **********************************************************/
  private boolean getFileWildcard( String selectedfile, Pattern pattern ) {
    boolean getIt = true;
    // First see if the file matches the regular expression!
    if ( pattern != null ) {
      Matcher matcher = pattern.matcher( selectedfile );
      getIt = matcher.matches();
    }

    return getIt;
  }

  private boolean deleteOrMoveFiles( SFTPv3Client sftpClient, String filename, String destinationFolder ) {
    boolean retval = false;

    // Delete the file if this is needed!
    if ( afterFtpPut.equals( "delete_file" ) ) {
      try {
        sftpClient.rm( filename );
        retval = true;
        if ( log.isDetailed() ) {
          logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.DeletedFile", filename ) );
        }
      } catch ( Exception e ) {
        logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.Error.CanNotDeleteRemoteFile", filename ), e );
      }

    } else if ( afterFtpPut.equals( "move_file" ) ) {
      String DestinationFullFilename = destinationFolder + Const.FILE_SEPARATOR + filename;
      try {
        sftpClient.mv( filename, DestinationFullFilename );
        retval = true;
        if ( log.isDetailed() ) {
          logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.DeletedFile", filename ) );
        }
      } catch ( Exception e ) {
        logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.Error.MovedFile", filename, destinationFolder ), e );
      }

    }
    return retval;
  }

  /**
   * copy a directory from the remote host to the local one.
   *
   * @param sourceLocation
   *          the source directory on the remote host
   * @param targetLocation
   *          the target directory on the local host
   * @param sftpClient
   *          is an instance of SFTPv3Client that makes SFTP client connection over SSH-2
   * @return the number of files successfully copied
   * @throws Exception
   */
  @SuppressWarnings( "unchecked" )
  private void GetFiles( String sourceLocation, String targetLocation, SFTPv3Client sftpClient, Pattern pattern,
    Job parentJob ) throws Exception {

    String sourceFolder = ".";
    if ( !Utils.isEmpty( sourceLocation ) ) {
      sourceFolder = sourceLocation + FTPUtils.FILE_SEPARATOR;
    } else {
      sourceFolder += FTPUtils.FILE_SEPARATOR;
    }

    Vector<SFTPv3DirectoryEntry> filelist = sftpClient.ls( sourceFolder );

    if ( filelist != null ) {
      Iterator<SFTPv3DirectoryEntry> iterator = filelist.iterator();

      while ( iterator.hasNext() && !parentJob.isStopped() ) {
        SFTPv3DirectoryEntry dirEntry = iterator.next();

        if ( dirEntry == null ) {
          continue;
        }

        if ( dirEntry.filename.equals( "." )
          || dirEntry.filename.equals( ".." ) || isDirectory( sftpClient, sourceFolder + dirEntry.filename ) ) {
          continue;
        }

        if ( getFileWildcard( dirEntry.filename, pattern ) ) {
          // Copy file from remote host
          copyFile(
            sourceFolder + dirEntry.filename, targetLocation + FTPUtils.FILE_SEPARATOR + dirEntry.filename,
            sftpClient );
        }

      }
    }
  }

  /**
   * copy a directory from the remote host to the local one recursivly.
   *
   * @param sourceLocation
   *          the source directory on the remote host
   * @param targetLocation
   *          the target directory on the local host
   * @param sftpClient
   *          is an instance of SFTPv3Client that makes SFTP client connection over SSH-2
   * @return the number of files successfully copied
   * @throws Exception
   */
  private void copyRecursive( String sourceLocation, String targetLocation, SFTPv3Client sftpClient,
    Pattern pattern, Job parentJob ) throws Exception {
    String sourceFolder = "." + FTPUtils.FILE_SEPARATOR;
    if ( sourceLocation != null ) {
      sourceFolder = sourceLocation;
    }

    if ( this.isDirectory( sftpClient, sourceFolder ) ) {
      Vector<?> filelist = sftpClient.ls( sourceFolder );
      Iterator<?> iterator = filelist.iterator();

      while ( iterator.hasNext() ) {

        SFTPv3DirectoryEntry dirEntry = (SFTPv3DirectoryEntry) iterator.next();

        if ( dirEntry == null ) {
          continue;
        }
        if ( dirEntry.filename.equals( "." ) || dirEntry.filename.equals( ".." ) ) {
          continue;
        }
        copyRecursive( sourceFolder + FTPUtils.FILE_SEPARATOR + dirEntry.filename, targetLocation
          + Const.FILE_SEPARATOR + dirEntry.filename, sftpClient, pattern, parentJob );
      }
    } else if ( isFile( sftpClient, sourceFolder ) ) {
      if ( getFileWildcard( sourceFolder, pattern ) ) {
        copyFile( sourceFolder, targetLocation, sftpClient );
      }
    }
  }

  /**
   * Checks if file is a file
   *
   * @param sftpClient
   * @param filename
   * @return true, if filename is a directory
   */
  public boolean isFile( SFTPv3Client sftpClient, String filename ) {
    try {
      return sftpClient.stat( filename ).isRegularFile();
    } catch ( Exception e ) {
      // Ignore errors
    }
    return false;
  }

  /**
   *
   * @param sourceLocation
   * @param targetLocation
   * @param sftpClient
   * @return
   */
  private void copyFile( String sourceLocation, String targetLocation, SFTPv3Client sftpClient ) {

    SFTPv3FileHandle sftpFileHandle = null;
    FileOutputStream fos = null;
    File transferFile = null;
    long remoteFileSize = -1;
    boolean filecopied = true;

    try {

      transferFile = new File( targetLocation );

      if ( ( onlyGettingNewFiles == false )
        || ( onlyGettingNewFiles == true ) && !FileExists( transferFile.getAbsolutePath() ) ) {

        new File( transferFile.getParent() ).mkdirs();

        remoteFileSize = this.getFileSize( sftpClient, sourceLocation );

        if ( log.isDetailed() ) {
          logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.ReceivingFile", sourceLocation, transferFile
            .getAbsolutePath(), "" + remoteFileSize ) );
        }

        sftpFileHandle = sftpClient.openFileRO( sourceLocation );

        fos = null;
        long offset = 0;

        fos = new FileOutputStream( transferFile );
        byte[] buffer = new byte[2048];
        while ( true ) {
          int len = sftpClient.read( sftpFileHandle, offset, buffer, 0, buffer.length );
          if ( len <= 0 ) {
            break;
          }
          fos.write( buffer, 0, len );
          offset += len;
        }
        fos.flush();
        fos.close();
        fos = null;

        nbfilestoget++;
        if ( remoteFileSize > 0 && remoteFileSize != transferFile.length() ) {
          filecopied = false;
          nbrerror++;
          logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.Error.RemoteFileLocalDifferent", ""
            + remoteFileSize, transferFile.length() + "", "" + offset ) );
        } else {
          nbgot++;
          if ( log.isDetailed() ) {
            logDetailed( BaseMessages.getString(
              PKG, "JobSSH2GET.Log.RemoteFileLocalCopied", sourceLocation, transferFile + "" ) );
          }
        }
      }
      // Let's now delete or move file if needed...
      if ( filecopied && !afterFtpPut.equals( "do_nothing" ) ) {
        deleteOrMoveFiles( sftpClient, sourceLocation, environmentSubstitute( destinationfolder ) );
      }

    } catch ( Exception e ) {
      nbrerror++;
      logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.Error.WritingFile", transferFile.getAbsolutePath(), e
        .getMessage() ) );
    } finally {
      try {
        if ( sftpFileHandle != null ) {
          sftpClient.closeFile( sftpFileHandle );
          sftpFileHandle = null;
        }
        if ( fos != null ) {
          try {
            fos.close();
            fos = null;
          } catch ( Exception ex ) {
            // Ignore errors
          }
        }
      } catch ( Exception e ) {
        // Ignore errors
      }

    }
  }

  private boolean CreateFolder( String filefolder ) {
    FileObject folder = null;
    try {
      folder = KettleVFS.getFileObject( filefolder, this );

      if ( !folder.exists() ) {
        if ( createtargetfolder ) {
          folder.createFolder();
          if ( log.isDetailed() ) {
            logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.FolderCreated", folder.toString() ) );
          }
        } else {
          return false;
        }

      }
      return true;
    } catch ( Exception e ) {
      logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.CanNotCreateFolder", folder.toString() ), e );

    } finally {
      if ( folder != null ) {
        try {
          folder.close();
        } catch ( Exception ex ) { /* Ignore */
        }
      }
    }
    return false;
  }

  /**
   * Create remote folder
   *
   * @param sftpClient
   * @param foldername
   * @return true, if foldername is created
   */
  private boolean CreateRemoteFolder( SFTPv3Client sftpClient, String foldername ) {
    boolean retval = false;

    if ( !sshDirectoryExists( sftpClient, foldername ) ) {
      try {
        sftpClient.mkdir( foldername, 0700 );
        retval = true;
        if ( log.isDetailed() ) {
          logDetailed( BaseMessages.getString( PKG, "JobSSH2GET.Log.RemoteFolderCreated", foldername ) );
        }

      } catch ( Exception e ) {
        logError( BaseMessages.getString( PKG, "JobSSH2GET.Log.Error.CreatingRemoteFolder", foldername ), e );
      }
    }
    return retval;
  }

  @Override
  public boolean evaluates() {
    return true;
  }

  @Override
  public List<ResourceReference> getResourceDependencies( JobMeta jobMeta ) {
    List<ResourceReference> references = super.getResourceDependencies( jobMeta );
    if ( !Utils.isEmpty( serverName ) ) {
      String realServerName = jobMeta.environmentSubstitute( serverName );
      ResourceReference reference = new ResourceReference( this );
      reference.getEntries().add( new ResourceEntry( realServerName, ResourceType.SERVER ) );
      references.add( reference );
    }
    return references;
  }

  @Override
  public void check( List<CheckResultInterface> remarks, JobMeta jobMeta, VariableSpace space,
    Repository repository, IMetaStore metaStore ) {
    JobEntryValidatorUtils.andValidator().validate( this, "serverName", remarks,
      AndValidator.putValidators( JobEntryValidatorUtils.notBlankValidator() ) );
    JobEntryValidatorUtils.andValidator().validate( this, "localDirectory", remarks,
      AndValidator.putValidators(
        JobEntryValidatorUtils.notBlankValidator(),
        JobEntryValidatorUtils.fileExistsValidator() ) );
    JobEntryValidatorUtils.andValidator().validate( this, "userName", remarks,
      AndValidator.putValidators( JobEntryValidatorUtils.notBlankValidator() ) );
    JobEntryValidatorUtils.andValidator().validate( this, "password", remarks,
      AndValidator.putValidators( JobEntryValidatorUtils.notNullValidator() ) );
    JobEntryValidatorUtils.andValidator().validate( this, "serverPort", remarks,
      AndValidator.putValidators( JobEntryValidatorUtils.integerValidator() ) );
  }

}
